package net.bwie.realtime.jtp.dwd.log.job;
import com.alibaba.fastjson.JSON;
import net.bwie.realtime.jtp.dwd.log.function.AdjustIsNewProcessFunction;
import net.bwie.realtime.jtp.dwd.log.function.AppLogSplitProcessFunction;
import net.bwie.realtime.jtp.utils.KafkaUtil;
import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.KeyedStream;
import org.apache.flink.streaming.api.datastream.SideOutputDataStream;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.ProcessFunction;
import org.apache.flink.util.Collector;
import org.apache.flink.util.OutputTag;

/**
 * DWD层数据应用开发：将ODS层采集原始日志数据，进行分类处理，存储Kafka队列。
 *      数据流向：kafka -> flink datastream -> kafka
 * @author xuanyu
 * @date 2025/5/18
 */
public class JtpAppLogEtlJob {
    public static void main(String[] args) throws Exception {
        // 1.执行环境env
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(1);
        // 2.数据源source
        DataStream<String> KafkaDataStream = KafkaUtil.consumerKafka(env, "topic-log");
        //KafkaDataStream.print("Kafka");
        // 3.数据转换transformation
        DataStream<String> pageStream = processLog(KafkaDataStream);
        // 4.数据输出sink
        KafkaUtil.producerKafka(pageStream,"dwd-traffic-page-log");
        // 5.触发执行execute
        env.execute("JtpAppLogEtlJob");
    }
    /**
     * 对实时获取APP流量日志数据进行ETL处理，并且进行分流
     *      1.数据清洗
     *      2.新老访客状态标记修复
     *      3.数据分流
     * @param stream app日志流
     */
    private static DataStream<String> processLog(DataStream<String> stream) {
        // 1.数据清洗
        DataStream<String> jsonStream = appLogCleaned(stream);
        // 2.新老访客状态标记修复
        DataStream<String> etlStream = processIsNew(jsonStream);
        // 3.数据分流
        DataStream<String> pageStream = splitStream(jsonStream);
        return pageStream;
    }

    private static DataStream<String> splitStream(DataStream<String> etlStream) {
        //侧边流输出标记
        final OutputTag<String> errorTag = new OutputTag<String>("error-log") {
        };
        final OutputTag<String> startTag = new OutputTag<String>("start-log") {
        };
        final OutputTag<String> displayTag = new OutputTag<String>("display-log") {
        };
        final OutputTag<String> actionTag = new OutputTag<String>("action-log") {
        };
        //日志分流处理
        SingleOutputStreamOperator<String> pageStream = etlStream.process(
                new AppLogSplitProcessFunction(errorTag, startTag, displayTag, actionTag)
        );
        //测流输出
        SideOutputDataStream<String> errorStream = pageStream.getSideOutput(errorTag);
        KafkaUtil.producerKafka(errorStream,"dwd-traffic-error-log");
        SideOutputDataStream<String> startStream = pageStream.getSideOutput(startTag);
        KafkaUtil.producerKafka(startStream,"dwd-traffic-start-log");
        SideOutputDataStream<String> displayStream = pageStream.getSideOutput(displayTag);
        KafkaUtil.producerKafka(displayStream,"dwd-traffic-display-log");
        SideOutputDataStream<String> actionStream = pageStream.getSideOutput(actionTag);
        KafkaUtil.producerKafka(actionStream,"dwd-traffic-action-log");
        return pageStream;
    }

    /**
     * 新老访客状态标记修复
     * @param jsonStream
     * @return
     */
    private static DataStream<String> processIsNew(DataStream<String> jsonStream) {
        //按照设备id分组
        KeyedStream<String, String> midStream = jsonStream.keyBy(
                new KeySelector<String, String>() {
                    @Override
                    public String getKey(String s) throws Exception {
                        /**
                         * "common": {
                         *     "ar": "500000",
                         *     "ba": "Xiaomi",
                         *     "ch": "xiaomi",
                         *     "is_new": "0",
                         *     "md": "Xiaomi Mix2 ",
                         *     "mid": "mid_570767",
                         *     "os": "Android 11.0",
                         *     "uid": "913",
                         *     "vc": "v2.1.132"
                         *   },
                         */
                        //需解析两次获取mid值
                        return JSON.parseObject(s).getJSONObject("common").getString("mid");
                    }
                }
        );
        //状态编程,对is_new效验修复
        DataStream<String> isNewStream = midStream.process(new AdjustIsNewProcessFunction());
        //返回数据流
        return isNewStream;
    }

    /**
     * 哪个
     * 数据清洗:将不合格的数据输出侧边流
     * @param stream
     * @return
     */
    private static DataStream<String> appLogCleaned(DataStream<String> stream) {
        //脏数据侧边流输出标记
        final OutputTag<String> dirtyTag = new OutputTag<String>("dirty-log"){};
        //数据清洗处理
        SingleOutputStreamOperator<String> cleanedStream = stream.process(new ProcessFunction<String, String>() {
            @Override
            public void processElement(String value,Context context, Collector<String> collector) throws Exception {
                try {
                    //解析JSON数据
                    JSON.parseObject(value);
                    //没有异常,解析正确,正常输出
                    collector.collect(value);
                }catch (Exception e){
                    //捕获异常,侧边流输出数据
                    context.output(dirtyTag,value);
                }
            }
        });
        //侧边流输出脏数据
        SideOutputDataStream<String> dirtyStream = cleanedStream.getSideOutput(dirtyTag);
        KafkaUtil.producerKafka(dirtyStream,"dwd-traffic-dirty-log");
        return cleanedStream;
    }
}